package com.my_test.webmagic.webmagic_modules;

import com.alibaba.fastjson.JSONObject;
import com.my_test.webmagic.module.entity.*;
import com.my_test.webmagic.module.service.*;
import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import javax.annotation.Resource;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.*;
import java.util.stream.Collectors;

/**
 * @author zhangyalin@act-telecom.com
 * @date 2019-05-23 10:13
 **/
@Component
public class SearchWebMagicProcessor implements PageProcessor {

    @Resource
    private TiebaTypeService tieziTypeService;
    @Resource
    private TieziService tieziService;
    @Resource
    private UserService userService;
    @Resource
    private FloorService floorService;
    @Resource
    private FloorLoService floorLoService;
    @Resource
    private TiebaService tiebaService;
    @Resource
    private TransportClient transportClient;

    //http://tieba.baidu.com/f/search/res?ie=utf-8&qw=%E6%97%B6%E9%97%B4
    //http://tieba.baidu.com/f/search/fm?ie=UTF-8&qw=%E6%97%B6%E9%97%B4
    //https://www.baidu.com/p/%CA%B1%BC%E4?from=tieba

    // 抓取网站的相关配置，包括编码、抓取间隔、重试次数等
    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setCharset("utf-8");
    public final static String regex1 = "http[s]?://tieba.baidu.com/f/search/res\\?[\\S]*&qw=[\\S]+";
    public final static String regex2 = "http[s]?://tieba.baidu.com/f/search/fm\\?[\\S]*&qw=[\\S]+";
    public final static String regex3 = "http[s]?://www.baidu.com/p/[\\S]+\\?from=tieba";


    @Override
    public void process(Page page) {

        String url = page.getUrl().get();
        System.out.println(url);
        List<String> all = page.getHtml().links().all();
        List<String> regexAll = new ArrayList<>();
        Document document = page.getHtml().getDocument();
        if (url.matches(regex1)) {//搜贴
            String keyword;
            if (url.contains("&rn")) {
                keyword = url.substring(url.indexOf("&qw=") + 4, url.indexOf("&rn="));
            } else {
                keyword = url.substring(url.indexOf("&qw=") + 4);
            }
            try {
                keyword = URLDecoder.decode(keyword, "utf-8");
            } catch (UnsupportedEncodingException e) {

            }
            List<SearchTiezi> searchTiezis = new ArrayList<>();
            Elements sPost = document.getElementsByClass("s_post");
            for (Element element : sPost) {
                Element element1 = element.getElementsByClass("p_title").first().getElementsByTag("a").first();
                if (element1.hasClass("bluelink") && element1.hasAttr("data-tid")) {
                    //表示是帖子
                    SearchTiezi searchTiezi = new SearchTiezi();
                    searchTiezi.setWord(keyword);
                    searchTiezi.setUrl(element1.absUrl("href"));
                    searchTiezi.setTitle(element1.text());
                    searchTiezi.setTieba(element.getElementsByClass("p_forum").first().getElementsByTag("font").first().text());
                    searchTiezi.setTiebaUrl(element.getElementsByClass("p_forum").first().absUrl("href"));
                    searchTiezi.setAuthorUsername(element.getElementsByTag("a").last().getElementsByTag("font").first().text());
                    searchTiezi.setAuthorUrl(element.getElementsByTag("a").last().absUrl("href"));
                    searchTiezi.setCreateTime(formatDate(element.getElementsByClass("p_date").first().text()));
                    searchTiezis.add(searchTiezi);
                }
            }
            page.putField("type", "searchTiezi");
            page.putField("entityList", searchTiezis);
            Elements pageEls = document.getElementsByClass("pager-search");
            if (pageEls.size() != 0) {
                Elements nextEls = pageEls.first().getElementsByClass("next");
                if (nextEls.size() != 0) {
                    page.addTargetRequest(nextEls.first().absUrl("href"));
                }
            }

        } else if (url.matches(regex2)) {//搜吧
            String keyword;
            if (url.contains("&rn")) {
                keyword = url.substring(url.indexOf("&qw=") + 4, url.indexOf("&rn="));
            } else {
                keyword = url.substring(url.indexOf("&qw=") + 4);
            }
            try {
                keyword = URLDecoder.decode(keyword, "utf-8");
            } catch (UnsupportedEncodingException e) {

            }
            List<SearchTieba> searchTiebas = new ArrayList<>();
            Elements forumEls = document.getElementsByClass("forum-item");
            for (Element element : forumEls) {
                SearchTieba tieba = new SearchTieba();
                tieba.setTiebaName(element.getElementsByClass("forum-name-wraper").first().text());
                tieba.setUrl(element.getElementsByClass("forum-name-wraper").first().absUrl("href"));
                tieba.setWord(keyword);
                searchTiebas.add(tieba);
            }
            page.putField("type", "searchTieba");
            page.putField("entityList", searchTiebas);
            Elements pageEls = document.getElementsByClass("pager-search");
            if (pageEls.size() != 0) {
                Elements nextEls = pageEls.first().getElementsByClass("next");
                if (nextEls.size() != 0) {
                    page.addTargetRequest(nextEls.first().absUrl("href"));
                }
            }
        } else if (url.matches(regex3)) {
            String keyword = url.substring(url.indexOf("p/") + 2, url.indexOf("?from="));
            try {
                keyword = URLDecoder.decode(keyword, "utf-8");
            } catch (UnsupportedEncodingException e) {

            }
            SearchUser user = new SearchUser();
            String urlText = document.getElementsByClass("url-desc").first().text();
            user.setWord(keyword);
            user.setUrl(urlText.substring(urlText.indexOf("：") + 1));
            user.setDetailUrl(document.getElementsByClass("detail-link").first().absUrl("href"));
            page.putField("type", "searchUser");
            page.putField("entity", user);
        }

    }


    @Override
    public Site getSite() {
        return site;
    }

    public static Date formatDate(String date) {
        DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm");
        return dateTimeFormatter.parseDateTime(date).toDate();
    }


    public static void main(String[] args) {

    }


}
